package main;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Scanner;
import java.util.Set;

import nmf.FeatureSparsenessNMFTool;
import nmf.NMFTool;
import nmf.NMFTool.Results;
import nmf.TopWords;

import org.ejml.simple.SimpleMatrix;

import svm.matrixToFile;
import svm.run_svm;
import utils.ClassificationUtil;

public class ReutersTest {
   
   public static String breakRegEx = "[.[^a-zA-Z]]";
   private static String TOPICS_START = "<TOPICS>";
   private static String TOPICS_END = "</TOPICS>";
   
   //Note args[0] will be the number of documents to use as a training set.
   public static void main(String[] args) throws FileNotFoundException, IOException {
      
      String[] fileNames = { "documentCollections/72_DOCS_PET-CHEM_TIN_VEG-OIL-ORANGE_SORGHUM_YEN_MONEY-SUPPLY_SILVER_WHEAT_INCOME_RAPE-OIL_SUNSEED.txt" };//"documentCollections/25_NEW_DOCS_INCOME_MONEY-FX_SHIP"};//"documentCollections/25_NEW_DOCS_ALUM_HOG_YEN"};
      int[] actualClass = classifications(fileNames[0]);
      int[] trainClass = Arrays.copyOf(actualClass, Integer.parseInt(args[0]));//new int[Integer.parseInt(args[0])];
      //for (int i = 0; i < trainClass.length; i++)
      // trainClass[i] = actualClass[i];
      
      SimpleMatrix m = utils.ClassificationUtil. getPreProcessedMatrix(fileNames, false);
      
      //Perform NMF
      FeatureSparsenessNMFTool nmftool = new FeatureSparsenessNMFTool();
      //NMFTool nmftool = new ETool();
      int maxReducedInnerDimension = (int) Math.ceil((double) (m.numCols() * m.numRows()) / (m.numCols() + m.numRows()));
      Results res = nmftool.factor(m, maxReducedInnerDimension, trainClass);
      SimpleMatrix m_nmf = res.H;
      
      /*if (nmftool instanceof FeatureSparsenessNMFTool)
      for (int cat = 0; cat < ((FeatureSparsenessNMFTool)nmftool).numberOfCategories; cat++) {
         for (int relev = 0; relev < ((FeatureSparsenessNMFTool)nmftool).categoriesTopWords.get(0).length;relev++) {
            System.out.println(ClassificationUtil.rDc.getWordList().get(((FeatureSparsenessNMFTool)nmftool).categoriesTopWords.get(cat)[relev]));
         }
       System.out.println();
      }*/
      
      //System.out.println("Factored Matrix: ");
      //System.out.println("\n" + m_nmf.toString());
      
      // Get the number of training documents (they are at the start of
      // the file containing the documents.
      //System.out.printf("How many training documents are there? ");
      
      SimpleMatrix m_train_nmf = m_nmf.extractMatrix(0, m_nmf.numRows(), 0, trainClass.length);
      
      //System.out.println("Training Matrix: ");
      //System.out.println("\n" + m_train_nmf.toString());
      
      //for (int docNum = 0; docNum < 3; docNum++)
      //Summarizer.printTopN(m, t.getWordList(), 10, docNum);           
      
      write_file("src/svm/matrix_file_nmf", m_nmf.transpose());
      write_file("src/svm/matrix_file_train_nmf", m_train_nmf.transpose());
      
      matrixToFile mft = new matrixToFile("src/svm/matrix_file_train_nmf", true, trainClass);
      matrixToFile mfc = new matrixToFile("src/svm/matrix_file_nmf", false, actualClass);
      run_svm svm = new run_svm("svmTrain", "svmFile", "svmOut");
      
      mfc.deleteFiles();
      mft.deleteFiles();
      svm.deleteFiles();
      
      System.out.println("Actual Classifications(" + actualClass.length + "): \n");
      for (int i = 0; i < actualClass.length; i++) {
         System.out.print(new Double(actualClass[i]) + " ");
      }
      System.out.println("\n");
      
      System.out.println("Classifications from SVM with factoring: \n");
      FileInputStream in = new FileInputStream(new File("svmOut"));
      Scanner sc = new Scanner(in);
      
      int totalCorrectNMF = 0;
      double currentClass = 0;
      
      for (int i = 0; sc.hasNext(); i++) {
         currentClass = sc.nextDouble();
         if (currentClass == actualClass[i])
            totalCorrectNMF++;
         System.out.print(currentClass + " ");
      }
      System.out.println("(" + totalCorrectNMF + ")\n");
      
      // svm without factoring
      SimpleMatrix m_train = m.extractMatrix(0, m.numRows(), 0, trainClass.length);
      
      write_file("src/svm/matrix_file", m.transpose());
      write_file("src/svm/matrix_file_train", m_train.transpose());
      
      matrixToFile mft_2 = new matrixToFile("src/svm/matrix_file_train", true, trainClass);
      matrixToFile mfc_2 = new matrixToFile("src/svm/matrix_file", false, actualClass);
      run_svm svm_2 = new run_svm("svmTrain", "svmFile", "svmOut");
      
      mfc_2.deleteFiles();
      mft_2.deleteFiles();
      svm_2.deleteFiles();
      
      System.out.println("Classifications from SVM without factoring: \n");
      FileInputStream in_2 = new FileInputStream(new File("svmOut"));
      Scanner sc_2 = new Scanner(in_2);
      int totalCorrectNoFactor = 0;
      for (int i = 0; sc_2.hasNext(); i++) {
         currentClass = sc_2.nextDouble();
         if (currentClass == actualClass[i])
            totalCorrectNoFactor++;
         System.out.print(currentClass + " ");
      }
      
      System.out.println("\n");
      Integer[] topWords;
      if ((NMFTool) nmftool instanceof TopWords) {
         topWords = ((TopWords) (NMFTool) nmftool).getTopWords();
         List<String> words = ClassificationUtil.rDc.getWordList();
         System.out.println("Relevant words inserted into H: ");
         for (int i = 0; i < res.W.numCols(); i++)
            System.out.println(words.get(topWords[i]));
      }
      
      System.out.println("\n");
      System.out.println("NMF Initialization Method: " + nmftool);
      System.out.println("Training set size: " + Integer.parseInt(args[0]));
      System.out.println("Total number of documents: " + actualClass.length);
      System.out.println("Word Dimension: " + m.numRows());
      System.out.println("Reduced Inner Dimension: " + res.W.numCols() + "\n");
      System.out.println("NMF factored error: " + res.error);
      System.out.println("Number of correctly classified documents with NMF: " + totalCorrectNMF);
      System.out.println("Number of correctly classified documents without NMF: " + totalCorrectNoFactor);
      
   }
   
   public static Set<String> getStopWords() {
      HashSet<String> output = new HashSet<String>();
      output.add("a");
      output.add("the");
      output.add("i");
      
      return output;
   }
   
   synchronized public static void write_file(String filename, SimpleMatrix A) throws FileNotFoundException, IOException {
      FileWriter out = new FileWriter(new File(filename));
      for (int i = 0; i < A.numRows(); i++) {
         for (int j = 0; j < A.numCols(); j++) {
            out.write(String.valueOf(A.get(i, j)));
            if (j != A.numCols() - 1) {
               out.write(" ");
            }
         }
         out.write("\n");
      }
      
      out.close();
   }
   
   public static int[] classifications(String filename) throws FileNotFoundException
   {
      Scanner sc = new Scanner(new File(filename));
      String line;
      ArrayList<Integer> al = new ArrayList<Integer>();
      int id = 1;
      
      HashMap<String, Integer> map = new HashMap<String, Integer>();
      
      while (sc.hasNextLine()) {
         line = sc.nextLine();
         int startNdx;
         if ((startNdx = line.indexOf(TOPICS_START)) != -1) {
            int endNdx = line.indexOf(TOPICS_END);
            String topic = line.substring(startNdx + TOPICS_START.length(), endNdx);
            topic.toUpperCase();
            
            if (!map.containsKey(topic))
            {
               map.put(topic, id++);
            }
            
            al.add(map.get(topic));
         }
      }
      
      int[] rtn = new int[al.size()];
      for (int i = 0; i < rtn.length; i++) {
         rtn[i] = al.get(i);
      }
      
      return rtn;
   }
}
